use pagination in the json serialization migration
Albert Sun
1 changed files with 19 additions and 9 deletions
+ 19
- 9
db/migrate/20131223032112_switch_to_json_serialization.rb
@@ -47,20 +47,30 @@ class SwitchToJsonSerialization < ActiveRecord::Migration |
||
| 47 | 47 |
quoted_table_name = ActiveRecord::Base.connection.quote_table_name(table) |
| 48 | 48 |
fields = fields.map { |f| ActiveRecord::Base.connection.quote_column_name(f) }
|
| 49 | 49 |
|
| 50 |
- rows = ActiveRecord::Base.connection.select_rows("SELECT id, #{fields.join(", ")} FROM #{quoted_table_name}")
|
|
| 51 |
- rows.each do |row| |
|
| 52 |
- id, *field_data = row |
|
| 50 |
+ page_start = 0 |
|
| 51 |
+ page_size = 1000 |
|
| 52 |
+ page_end = page_start + page_size |
|
| 53 | 53 |
|
| 54 |
- yaml_fields = field_data.map { |f| from.load(f) }.map { |f| to.dump(f) }
|
|
| 54 |
+ begin |
|
| 55 |
+ rows = ActiveRecord::Base.connection.select_rows("SELECT id, #{fields.join(", ")} FROM #{quoted_table_name} WHERE id >= #{page_start} AND id < #{page_end}")
|
|
| 56 |
+ puts "Grabbing rows of #{table} from #{page_start} to #{page_end}"
|
|
| 57 |
+ rows.each do |row| |
|
| 58 |
+ id, *field_data = row |
|
| 55 | 59 |
|
| 56 |
- yaml_fields.map! {|f| f.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '??') }
|
|
| 60 |
+ yaml_fields = field_data.map { |f| from.load(f) }.map { |f| to.dump(f) }
|
|
| 57 | 61 |
|
| 58 |
- update_sql = "UPDATE #{quoted_table_name} SET #{fields.map {|f| "#{f}=?"}.join(", ")} WHERE id = ?"
|
|
| 62 |
+ yaml_fields.map! {|f| f.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '??') }
|
|
| 59 | 63 |
|
| 60 |
- sanitized_update_sql = ActiveRecord::Base.send :sanitize_sql_array, [update_sql, *yaml_fields, id] |
|
| 64 |
+ update_sql = "UPDATE #{quoted_table_name} SET #{fields.map {|f| "#{f}=?"}.join(", ")} WHERE id = ?"
|
|
| 61 | 65 |
|
| 62 |
- ActiveRecord::Base.connection.execute sanitized_update_sql |
|
| 63 |
- end |
|
| 66 |
+ sanitized_update_sql = ActiveRecord::Base.send :sanitize_sql_array, [update_sql, *yaml_fields, id] |
|
| 67 |
+ |
|
| 68 |
+ ActiveRecord::Base.connection.execute sanitized_update_sql |
|
| 69 |
+ end |
|
| 70 |
+ page_start += page_size |
|
| 71 |
+ page_end += page_size |
|
| 72 |
+ |
|
| 73 |
+ end until rows.count == 0 |
|
| 64 | 74 |
end |
| 65 | 75 |
|
| 66 | 76 |
end |